{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import gym\n",
    "#import math\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 1000\n",
    "THRESHOLD = 45\n",
    "MONITOR = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN():\n",
    "    def __init__(self, env_string,batch_size=64):\n",
    "        self.memory = deque(maxlen=100000)\n",
    "        self.env = gym.make(env_string)\n",
    "        input_size = self.env.observation_space.shape[0]\n",
    "        action_size = self.env.action_space.n\n",
    "        self.batch_size = batch_size\n",
    "        self.gamma = 1.0\n",
    "        self.epsilon = 1.0\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.995\n",
    "        \n",
    "        alpha=0.01\n",
    "        alpha_decay=0.01\n",
    "        if MONITOR: self.env = gym.wrappers.Monitor(self.env, '../data/'+env_string, force=True)\n",
    "        \n",
    "        # Init model\n",
    "        self.model = Sequential()\n",
    "        self.model.add(Dense(24, input_dim=input_size, activation='tanh'))\n",
    "        self.model.add(Dense(48, activation='tanh'))\n",
    "        self.model.add(Dense(action_size, activation='linear'))\n",
    "        self.model.compile(loss='mse', optimizer=Adam(lr=alpha, decay=alpha_decay))\n",
    "\n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def choose_action(self, state, epsilon):\n",
    "        if np.random.random() <= epsilon:\n",
    "            return self.env.action_space.sample()\n",
    "        else:\n",
    "            return np.argmax(self.model.predict(state))\n",
    "\n",
    "    def preprocess_state(self, state):\n",
    "        return np.reshape(state, [1, 4])\n",
    "\n",
    "    def replay(self, batch_size):\n",
    "        x_batch, y_batch = [], []\n",
    "        minibatch = random.sample(self.memory, min(len(self.memory), batch_size))\n",
    "        for state, action, reward, next_state, done in minibatch:\n",
    "            y_target = self.model.predict(state)\n",
    "            y_target[0][action] = reward if done else reward + self.gamma * np.max(self.model.predict(next_state)[0])\n",
    "            x_batch.append(state[0])\n",
    "            y_batch.append(y_target[0])\n",
    "        \n",
    "        self.model.fit(np.array(x_batch), np.array(y_batch), batch_size=len(x_batch), verbose=0)\n",
    "        #epsilon = max(epsilon_min, epsilon_decay*epsilon) # decrease epsilon\n",
    "       \n",
    "\n",
    "    def train(self):\n",
    "        scores = deque(maxlen=100)\n",
    "        avg_scores = []\n",
    "        \n",
    "\n",
    "        for e in range(EPOCHS):\n",
    "            state = self.env.reset()\n",
    "            state = self.preprocess_state(state)\n",
    "            done = False\n",
    "            i = 0\n",
    "            while not done:\n",
    "                action = self.choose_action(state,self.epsilon)\n",
    "                next_state, reward, done, _ = self.env.step(action)\n",
    "                next_state = self.preprocess_state(next_state)\n",
    "                self.remember(state, action, reward, next_state, done)\n",
    "                state = next_state\n",
    "                self.epsilon = max(self.epsilon_min, self.epsilon_decay*self.epsilon) # decrease epsilon\n",
    "                i += 1\n",
    "\n",
    "            scores.append(i)\n",
    "            mean_score = np.mean(scores)\n",
    "            avg_scores.append(mean_score)\n",
    "            if mean_score >= THRESHOLD and e >= 100:\n",
    "                print('Ran {} episodes. Solved after {} trials ✔'.format(e, e - 100))\n",
    "                return avg_scores\n",
    "            if e % 100 == 0:\n",
    "                print('[Episode {}] - Mean survival time over last 100 episodes was {} ticks.'.format(e, mean_score))\n",
    "\n",
    "            self.replay(self.batch_size)\n",
    "        \n",
    "        print('Did not solve after {} episodes 😞'.format(e))\n",
    "        return avg_scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Episode 0] - Mean survival time over last 100 episodes was 16.0 ticks.\n",
      "[Episode 100] - Mean survival time over last 100 episodes was 17.47 ticks.\n",
      "[Episode 200] - Mean survival time over last 100 episodes was 28.1 ticks.\n",
      "Ran 254 episodes. Solved after 154 trials ✔\n"
     ]
    }
   ],
   "source": [
    "env_string = 'CartPole-v0'\n",
    "agent = DQN(env_string)\n",
    "scores = agent.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXyV5Z338c/vZA8hgayEQAhL2FHAiAq4o+JSt27a2lq1Rac61emMWmeeedqZp8vYGWurY22xWq1al1qXuqGi4C4S9iXsEEL2QPY9OdfzRw4UMCEh5OTknHzfr9d5cc59tt/FHb5cue7rvm5zziEiIsHHE+gCRESkdxTgIiJBSgEuIhKkFOAiIkFKAS4iEqTC+/PLkpOTXVZWVn9+pYhI0Fu1alWFcy7l6O39GuBZWVnk5ub251eKiAQ9M8vvbLuGUEREgpQCXEQkSCnARUSClAJcRCRIKcBFRIKUAlxEJEgpwEVEglSPA9zMwsxsjZm95nv8uJntNrO1vttM/5UpIhKcymqb+OWSLewsr+vzzz6eHvjtQN5R2+50zs303db2YV0iIiFhW0kdv12+k9Kapj7/7B4FuJmNAi4F/tDnFYiIhLD8A/UAZCUN6fPP7mkP/NfAXYD3qO0/M7P1Zna/mUV19kYzW2RmuWaWW15efiK1iogEnb37G4gM9zAiPrrPP7vbADezy4Ay59yqo566B5gMnAokAnd39n7n3GLnXI5zLicl5QtrsYiIhLQ9++sZPTwGj8f6/LN70gOfB1xuZnuAZ4HzzOwp51yx69AM/BGY0+fViYgEufz9DX4ZPoEeBLhz7h7n3CjnXBZwDfCec+46M0sHMDMDrgQ2+qVCEZEg5Zxj74EGMpNi/fL5J7Kc7NNmlgIYsBa4pW9KEhEJDeV1zTS0tPutB35cAe6cWw4s990/zw/1iIiEjL37GwD81gPXmZgiIn6S7wvwgI2Bi4hI7+woryMizBg1PMYvn68AFxHxk+2ltYxNHkJEmH+iVgEuIiFvR1kt331iJfe8uIGS6r4/pb0r20rryE4b6rfPV4CLSMh7aNlOPthewYur93HLU6toaTv6pPK+19jSTkFlAxNTFeAiIr2yv66Z19cX8405mfz66zNZW1DFnz7d4/fv3VFWh3MwMS3Ob99xIvPARUQGvOdz99HS7uWbp2WSnTaUEfHRbCmp9ct3fbyjgh+9uJ5LZqST7et5ZyvARUSOX7vX8fSKfE4fl3hoLDp9WDRFVY19/l1/+nQPP/7bJobFRPD793eREBNBRJgxxk9TCEEBLiIh7P1tZeyrbOSei6cc2jZyWAybCqv75POdc3yycz+PfLiL5VvLWTAljd9cM5NnVxawdHMpE9Pi/DYDBRTgIhLCnv28gJShUVw4Le3QtoxhMbyzuRTnHB1LOR0f5xzby+p46rN83t9WTv7+BpLjIrnzokncfNY4wsM83DR/LDfNH9uXTemUAlxEQta20lpOG5t4RC94ZEI0LW1e9te3kBzX6WUMOrWpqJonPtnDe1vKqahrJircw5nZydx6zgQunzmS6IgwfzThmBTgIhKSnHMUVzdx4bQRR2xPH9ZxVmRxVVOPAnzZ1jJeXlPIq+uKiI0M5/wpqZwyZjiXzkgn6Tj+A/AHBbiIhKQD9S00t3lJTzjySjgZvgAvrGpkxqiELt9f3djK/3ttMy+s2kdCTATXz83ijgUTSYiJ8Gvdx0MBLiIhqdh3xmV6wpHrkIz0BfixZqK8s7mUe15cz/76Fv7xvAn84Pxsvx6M7C0FuIiEpIMBPXLYkT3w4bERRIV7KK4+MsC9XsfrG4p5aU0h720pY9rIeB6/YQ7TM7rupQeaAlxEQlJXPXAzI2NYDAUH/h7gBQcauO2ZNawrqCJjWAy3nTuBfzx/AlHh/X9g8ngowEUkJBVVNxIZ5iFpSOQXnpszNpEXVxeyKr+S19YX8ddV+3DAfV89matmZfjlAsT+oAAXkZBUXNXEiIToTsP49gXZvLy2kC8//AmRYR7OnZzCjy6ewthk/5016Q8KcBEJScXVjV+YgXJQekIMd100mTc2FPOLq2f4dclXf1KAi0hIKqpqYs7YxC6fv3H+WG7sh7Ml/WngzYsRETlBNU2tlNQ0MTrRPxcTHih6HOBmFmZma8zsNd/jsWa2wsy2m9lzZvbFIwUiIgHw8fYK2r2O+ROSA12KXx1PD/x2IO+wx/cC9zvnsoFK4Ka+LExEpLeWbS0jPjqc2ZnDAl2KX/UowM1sFHAp8AffYwPOA17wveQJ4Ep/FCgicjy8XseyreWcNTGF8AF49mRf6mnrfg3cBRy8kFwSUOWca/M93gdkdPZGM1tkZrlmllteXn5CxYqIdGdjUTXltc2cOyk10KX4XbcBbmaXAWXOuVWHb+7kpa6z9zvnFjvncpxzOSkpKb0sU0SkZ5ZsLCHMY5w/JfQDvCfTCOcBl5vZJUA0EE9Hj3yYmYX7euGjgCL/lSki0j3nHEs2lnDGuCSGxYb+vIpue+DOuXucc6Occ1nANcB7zrlvAsuAr/hedj3wit+qFBHpge1ldeyqqOei6SO6f3EIOJER/ruBH5rZDjrGxB/tm5JERHpnycYSzOCiqWndvzgEHNeZmM655cBy3/1dwJy+L0lEpHeWbCzhlMzhpMZ3fgp9qAntOTYiMmjs3d/A5uIaFg6S4RNQgItIiFiyqRiAi6YpwEVEgsqbG0uYnhEf8uufHE4BLiJBr6S6iTV7q7h4enqgS+lXCnARCXpvby4BBtfwCSjARSQEvLWphAmpcUxIjQt0Kf1KAS4iQa2xpZ2Vuys5d9LgW6pDAS4iQS03/wAt7V7mhvja351RgItIUPt4x34iwow5WV1fPi1UKcBFJKh9vKOCWaOHMyRq8F3iVwEuIkGruLqRjUXVnJk9+IZPQAEuIkHs9fXFOAeXnjS45n8fpAAXkaD12vpipqbHMy5lcE0fPEgBLiJBqeBAA2sLqrjs5MHZ+wYFuIgEqVfWFgLwpZNGBriSwFGAi0jQcc7x0ppC5mQlDqrFq46mABeRoLOhsJqd5fVcOSsj0KUElAJcRILOS2sKiQzzcOmMwTv+DQpwEQkybe1eXl1XxHmTU0mIjQh0OQGlABeRoPLhjgoq6lq4avbgHj4BBbiIBJmnPs1neGwE5wzC1QeP1m2Am1m0mX1uZuvMbJOZ/Ydv++NmttvM1vpuM/1frogMZhsLq3l3Sxk3zhtLVHhYoMsJuJ6s/tIMnOecqzOzCOAjM3vT99ydzrkX/FeeiMjf/e97OxgaHc7187ICXcqA0G0P3HWo8z2M8N2cX6sSETnK1pJalmwq4Ya5WcRHD+6Dlwf1aAzczMLMbC1QBrzjnFvhe+pnZrbezO43s6gu3rvIzHLNLLe8vLyPyhaRwebB97YzJDKMG+ePDXQpA0aPAtw51+6cmwmMAuaY2XTgHmAycCqQCNzdxXsXO+dynHM5KSk66CAix6+wqpE3NhRz3eljGBYbGehyBozjmoXinKsClgMLnXPFvuGVZuCPwBw/1CciwrOf78UB150+JtClDCg9mYWSYmbDfPdjgAXAFjNL920z4Epgoz8LFZHBqaXNy7MrCzh3UuqgXvekMz2ZhZIOPGFmYXQE/vPOudfM7D0zSwEMWAvc4sc6RWSQeuTDXZTXNnODZp58QbcB7pxbD8zqZPt5fqlIRMRnV3kdv1m6nUtnpHNmto6hHU1nYorIgPXQsp2EeYwfXz410KUMSApwERmQSqqb+Nu6Qr6WM4rUodGBLmdAUoCLyID0yIe7aPc6vnvmuECXMmApwEVkwNlX2cCTn+Zz9exRmnlyDApwERlQnHP87PU8zOCHF0wMdDkDWk+mEYqI+F1dcxt3/mUdDS3tvL+tnLsXTmbksJhAlzWgKcBFZEB4cfU+3txYQnSEh2+elsktZ2vsuzsKcBEJOOccT32Wz/SMeF69bT4dJ3hLdzQGLiIBl5tfybbSOq47bYzC+zgowEUk4J76LJ+h0eFcPnNkoEsJKgpwEQmo/XXNvLmhhC/PHkVspEZ1j4cCXEQC6vncfbS0e/nGaZmBLiXoKMBFJGC8XsefP89nzthEJqYNDXQ5QUcBLiIB88H2cgoONOpCDb2kABeRgHnqs70kDYlk4bQRgS4lKCnARSQgCqsaeW9LKV87dTSR4Yqi3tDfmogExMHrXH5jjg5e9pYCXET6XVu7l+dWFnDOxBStNngCFOAi0u/e31ZOWW0z16j3fUIU4CLS715YtY/EIZGcOyk10KUENQW4iPSrA/UtLM0r5cqZGTp4eYK6/dszs2gz+9zM1pnZJjP7D9/2sWa2wsy2m9lzZhbp/3JFJNi9sraQ1nbHV3NGBbqUoNeT//6agfOccycDM4GFZnY6cC9wv3MuG6gEbvJfmSISKv6Su4/pGfFMSY8PdClBr9sAdx3qfA8jfDcHnAe84Nv+BHClXyoUkZCxqaiazcU1fPWU0YEuJST0aADKzMLMbC1QBrwD7ASqnHNtvpfsAzK6eO8iM8s1s9zy8vK+qFlEgtTTK/YSFe7hCi0b2yd6FODOuXbn3ExgFDAHmNLZy7p472LnXI5zLiclJaX3lYpIUKtpauXlNYVcfvJIhsXqkFlfOK5DwM65KmA5cDowzMwOLt47Cijq29JEJJS8tLqQhpZ2vn1GVqBLCRk9mYWSYmbDfPdjgAVAHrAM+IrvZdcDr/irSBEJfm9tKmFiWhwzRiUEupSQ0ZMeeDqwzMzWAyuBd5xzrwF3Az80sx1AEvCo/8oUkWBW19zGyj0HdOJOH+v2+kXOufXArE6276JjPFxE5Jg+3lFBa7vjHAV4n9JpUCLid8u3lhEXFU5O1vBAlxJSFOAi4lfOOZZtKefM7GQiwhQ5fUl/myLiV1tKaimpadL4tx8owEXEr5ZtLQPg7Ek6D6SvKcBFxG+cc7yXV8bU9HjS4qMDXU7IUYCLiN/87v1d5OZXcrlOnfeLbqcRiogcr9KaJu57eyvP5+7j8pNHsujMcYEuKSQpwEWkz+yva+aXS7bywup9OOe49dzx3LFgIh6PBbq0kKQAF5ETVtPUygu5+/jNu9upb27jW6eP4Ttzs8hKHhLo0kKaAlxEeq2kuonnVhaw+IOd1Le0c8a4JP7zimlkpw0NdGmDggJcJIQ0tbbz1qYStpXWMn1kAhfPSO/z72hua2f51nKe/DSfj3ZUALBw2gj+4ZzxnDQqATMNl/QXBbhIkGluaycqPIyaplaqG1qJDPcQFe5hQ2E1//TcWirqWgAYGhXO+VPS+uTCwXXNbazZW8mr64p4c2MJtU1tpMVH8c8XTOTiGSOYkKoedyAowEUGuCc/y2dNfiVmxs7yOtYWVDEueQh7DzTQ5j3yOirZqXE8cM0sapvbuPnJVazYvZ8zs3t3Ak3+/noWf7CL97eVs6+yEYC4qHAunJbG5SePZN4EnRofaApwkQHs7U0l/PvLG0mOiyIyzEiJj2bRWePYWlLLgqlpTEiNo7XdS1OrlzCDr+SMJi4qnMaWdqIjPLyzubTHAV5c3cjq/CpW7jnAh9vL2VleT2SYhwVTU7nm1NFMHRnP3PHJREeE+bnV0lMKcJEBal9lA/e8uIGp6fG8fOu84xoKiYkMY/6EFJZuLuUnX5rW5TS+2qZWnvpsLy+sKmBneT0A0REeThubxLVzMrn0pHTSE2L6pD3S9xTgIgNQWW0TN/xxJS3tXh64dmavxrG/dHI6S/NKWba1jPOnpB3a7vU6Vu45wMtrC3ltfTG1TW3MyUrk2jmZnJqVyJT0+D4ZNxf/U4CLDDDrCqq46Ylc6ppbeew7p/b6AOElM9K5980t/P6DXZw/JY3tpbW8tKaQV9YWUVjVSGxkGAunjeA787I4adSwPm6F9AcFuMgAUnCggRsfX0lMZBhPf3c+k0b0fnZHRJiHG+eP5aev55Hz03eoqGshzGOclZ3MXQsnccHUNGIjFQHBTHtPZICoaWrlxsdX0tru5bkbzmBCatwJf+b1c7MI8xgb9lVz0qgELjt5JMlxUX1QrQwECnCRAcDrdfzgmTXsrqjnTzfN6ZPwho5e+A3zxvbJZ8nAoyMVIgPAnz7dw/Kt5fz48mnMHZ8c6HIkSHQb4GY22syWmVmemW0ys9t9239iZoVmttZ3u8T/5YqEnoIDDfzizS2cNzmV607LDHQ5EkR6MoTSBvyzc261mQ0FVpnZO77n7nfO/Y//yhMJfT99fTMeM3521XStIyLHpdsAd84VA8W++7Vmlgdk+LswkcHgk50VvLWplDsvmqQTZuS4HdcYuJllAbOAFb5Nt5nZejN7zMyGd/GeRWaWa2a55eXlJ1SsSChxznHvkq2kJ0Rz03wdaJTj1+MAN7M44K/AHc65GuBhYDwwk44e+n2dvc85t9g5l+Ocy0lJ0VWpRQ5atrWMdQVV3LEgW+uLSK/0KMDNLIKO8H7aOfcigHOu1DnX7pzzAo8Ac/xXpkjo+XTnfiLDPVw9e1SgS5Eg1ZNZKAY8CuQ553512PbDV4q/CtjY9+WJhK4dZXWMT4nTkqzSaz2ZhTIP+BawwczW+rb9K3Ctmc0EHLAHuNkvFYqEqO1ldczK7PTQkUiP9GQWykdAZ3Ob3uj7ckQGh8aWdgqrGvlazuhAlyJBTL+7iQTAzvI6nKPPTpmXwUkBLhIAO8rqgI5LoIn0lgJcJAB2lNUR5jHGJA0JdCkSxBTgIgHw6a79ZKfG6co3ckL00yPSz7aU1LAqv5Iva/63nCAFuEg/cs7x+Md7iAz38JVTFOByYnRBB5EutHsdecU1hHmMyoYWEodEkjo0mvjocMKP8+SbtnYv20rrePC97by5sYRvnpbJ8CGRfqpcBgsFuEgXHvtoNz97I+8L24dGhXP+lFQunDaCzMRYhsVGEHPYWiat7Y6d5XW8s7mUirpmiqubWLO3Eq+D6AgPd140iVvOHt+fTZEQpQAX6URbu5fHP9nDrMxhLDpzHAkxEVTUt7C/rpnNRTW8k1fKy2uLjvkZ0REeRibEMDQ6nJvPHs+YxFgumjZCPW/pMwpwkU4szSujsKqRf79sKgunj/jC863tXjYUVlNe20xVQwvNbd5Dz3nMGJ0YyyljhhMXpX9i4j/66RLpxDubS0kaEsmCKamdPh8R5mG21jGRANMsFJFOFFc3kpkUe9wHK0X6k346RTpRUtNEekJ0oMsQOSYFuEgnSqubSItXgMvApgAXOUptUyv1Le2MUIDLAKcAFzlKaU0TACM0hCIDnAJc5Cgl1c0AGkKRAU8BLnKU4upGAB3ElAFPAS5ylINDKOqBy0CnABc5SklNE8NiI4g+bH0TkYGo2wA3s9FmtszM8sxsk5nd7tueaGbvmNl23586LU1CQkl1s2agSFDoSQ+8Dfhn59wU4HTgVjObCvwIeNc5lw2863ssQeZAfQs5P13KST95i399aQP1zW2BLimgnHPsqqjT+LcEhW4D3DlX7Jxb7btfC+QBGcAVwBO+lz0BXOmvIsV/3txYTEVdM3PHJ/PM53u55IEPWZVfGeiyAmb13ip2ldezYGpaoEsR6dZxjYGbWRYwC1gBpDnniqEj5IHOV/2RAe319cWMSx7Cw9fN5tnvnU5bu+Orv/uEX729ldZ2b/cfEGKe/iyfuKhwrpyZEehSRLrV4wA3szjgr8Adzrma43jfIjPLNbPc8vLy3tQoflJR18xnu/Zz6UnpmBmnjUvizTvO5KpZo3jgvR1c/dtP2FZaG+gy+01pTROvbSjmqlkZDNEysBIEehTgZhZBR3g/7Zx70be51MzSfc+nA2Wdvdc5t9g5l+Ocy0lJSemLmqWPfLpzP14HF079+3rX8dER3Pe1k/nddbPZV9nApQ98yK+XbqOlLfR74w8v30m71/G9M8cFuhSRHunJLBQDHgXynHO/OuypvwHX++5fD7zS9+WJP+2uqAcgOy3uC88tnJ7O0h+ezSUz0vn10u1c9uCHrN4bumPje/c38OfP9/Ll2RlkJsUGuhyRHulJD3we8C3gPDNb67tdAvwXcIGZbQcu8D2WILK7op6RCdFdzndOioviN9fM4rHv5FDX1MaXH/6E/3h1E7VNrf1cqX81tLSx6MlcosM9/OD87ECXI9Jj3Q70Oec+AqyLp8/v23KkP+2uqGdsypBuX3fe5DTe/mESv1yyhcc/2cOr64r4lwsn8dWc0YR5uvrRCB6PfLCbLSW1PHHjHEYNV+9bgofOxBzEdlfUMza5+wAHiIsK5z+vmM7fbp1PVtIQfvTiBr704Ed8srPCz1X6V3NbO09+ls+5k1I4e6KO0UhwUYAPUpX1LVQ3tpKV1LMAP2jGqAT+cssZPHjtLKobW/nGIyv49mOfs7Gw2k+V+tfzKwuoqGvmhnljA12KyHHTXKlBapfvAOa4HgyhHM3M+NLJI7lgahpPfprPQ8t3cNmDH7FgSiq3njuBWUFwsd+m1nb+682OIaHZmcM4Mzs50CWJHDcF+CC1xxfgx9sDP1x0RBjfO2scX58zmj9+tIc/frKbq377CfMmJHHbudmcPi6RjklMA0dTaztLNpbwm3e3s7uinhvmZXH3wskDrk6RnlCAD1Ibi6qJDPMwOvHED9rFR0dw+4JsbjpzLH9ekc/iD3Zz7SOfccqY4dx27gTOmZQS0IBsam3n/W3lvLGhmHfzyqhrbmNiWhxP3jSHM7M17i3BSwE+CHm9jrc2lnDWxGQiwvruMEhcVDiLzhrPt8/I4vncAn7//i5ueHwl00bGc+u5E1g4bQSefpy10truZWNhNbc/u5a9BxoYHhvBZSelc8mMdOZPSO7XWkT8QQE+CK3dV0VRdRN3Lpzkl8+Pjgjj22dkcc2pmby8tpCHl+/k+0+vZkJqHN8/ZzyXnzyS8D78j+OgwqpGKutbqGlq5eU1hby8toiWNi9p8VE8en0OZ01M6dP/sEQCTQE+CL26rojIMA/nT/HvinuR4R6+ljOaL88exRsbinlo2Q5++Pw67l+6jZvPGs+VszKIO8E1R5xzbCis5udv5PHZrgOHtkdHePjy7FGcNCqBC6amkRwXdaLNERlwFOCDTFltE8+tLOCi6SOIj47ol+8M83TMWrl0Rjrvbinjf5ft4P+8vJGfvZ7Hwukj+MopozhjXFKPhzT+9aUNvLymEICocA+VDa0kx0Vy50WTGJ8yhHCPhzPGJ2lBKgl5+gkfZB54dzstbV5+eMHEfv9uj8e4YGoaC6aksnpvFX9dvY9X1xXx0ppCosI9ZAyLIS0+mugID9fMyWTehGSiwj1HDHuU1zbz3MoCThubyJT0eBpa2pg8Ip4rZ2WQENM//yGJDBQK8EFkV3kdz3xewDfmZPb4DEx/MDNOGTOcU8YM5/9eNpWleaWs31dNYWUjpTVNFFQ2cPOTq4CO3vspY4bz4LWzSIuP5tV1RbR7HT+5fBoT04YGrA0iA4ECfBD577e2DrgFm6IjwrjspJFcdtLIQ9ta2728vKaQA76zRR//ZA+XPvAhE9OGsqOsjukZ8QpvERTgg8bqvZW8ubGEOxZkkzJ0YB/Qiwjz8NWc0YceXzw9nYeW7aC8rpn4mAhuOXt8AKsTGTgU4IOAc47/emMLyXGRfDcIL1YwY1QCv/vWKYEuQ2TA0aTYQeDdvDI+33OA28/PPuFpeyIycCjAQ1y713Hvki2MTR7CNXMyA12OiPQhBXiIe3blXraX1XHnRZN0FqJIiAmKf9EFBxr4fPeB7l8oR9hTUc/PXs/jjHFJXDx9RPdvEJGgEhQB/vD7O7n5ydxAlxFUnHPc/df1hHuM+752spZLFQlBQRHgmYmxVDa0UhNiF9P1p7c2lbBi9wHuWjiZkcNiAl2OiPhBUAT4GN+a1Xv3NwS4kuDg9TruXbKViWlxXHPq6O7fICJBqdsAN7PHzKzMzDYetu0nZlZoZmt9t0v8WeTBiw4UHFCA98TybWXsrqjntvOy/bJsq4gMDD351/04sLCT7fc752b6bm/0bVlHykzy9cB7EeDltc19Xc6A98Qn+aQOjWLhNB24FAll3Qa4c+4DIKBTQOKjIxgWG3HMAN9ZXodz7ohtGwurmfPzpbyyttDfJQ4YW0pqeH9bOd84LZPIcPW+RULZifwLv83M1vuGWLq8DLmZLTKzXDPLLS8v7/WXjUmM7TLAt5fWcv5977NkY8kR21/fUIxzHYs4Nbe19/q7g8n/vLWVodHhfGduVqBLERE/622APwyMB2YCxcB9Xb3QObfYOZfjnMtJSen9BWRHHyPA9/gObi7ZdGSAL91cSsrQKPZVNvLMir29/u5gsa6giqV5Zdxy9niGxUYGuhwR8bNeBbhzrtQ51+6c8wKPAHP6tqwvykyMpbCykbZ27xeeK6ttAmD51vJDz++pqGd7WR3/cPZ4zhiXxIPv7aCuuc3fZQbU0yvyiY0M43r1vkUGhV4FuJmlH/bwKmBjV6/tK5mJsbR5HcXVTV94rrSm40BldWMrufmVACzfWgbAgilp3LVwEvvrW1j8wS5/lxkwdc1tvLa+mMtOSteCVSKDRE+mET4DfApMMrN9ZnYT8Esz22Bm64FzgX/yc51kHmMqYVlNE0Ojw4kK9/DM5x1DJSvzK8kYFkNmUiyzMofzpZNH8ttlO1i9t/ILBztDwavrimhoaefrmvctMmh021Vzzl3byeZH/VDLMR0+lXDuUc+V1TaTmRjLOZNSeGjZTq6fm0XungOcNjbp0Gt+esV0VudXcvVvP2FIZBhnT0rh7oWTGZMUuEuL9ZW2di+/f38nU9PjmZ3Z5fFkEQkxQTPPLD0hhnCPke/rgXu9jnZvR0+6tKaJ1KFRfP+cCaQOjeKfn19HaU0zOVl/D7OE2Aj+/L3TuPOiSVw+M4MPt1dw+f9+zKc79wekPX3pb+uK2LO/gR+cn601T0QGkaAJ8DCPMWp4zKGZKIueXMWdL6wDOnrgafHRDIkK566Fk9ldUQ9AzpjEIz5jTNIQbj13Ar+4egav/+OZpA6N4jt//JyPtlf0b2P6UHltMz9/YwtT0uO5cGpaoMsRkX4UNAEOHVMJD46Bry2o4p3NpTS3tVNR10xqfDQAV8/K4ORRCcRHhzNpRNcXvs1Mio/8NQQAAAdrSURBVOW5m89gbPIQvv/0KvZVBt9p+tUNrdz69Gpqm1q5/+sn4/Go9y0ymARVgI9J6pgL3tjSEdq1TW18uK0C5yDVd6Fej8dY/O0c/vy90wnrJtASh0Ty+2+dgtfBN/+wgj98uOvQsMxAV1bTxBUPfcSagkp++ZWTmDwiPtAliUg/C6oAz0yMpaqhlc3FNYe2vbKuCIA0Xw/84P3pGQk9+swxSUP43XWnMDw2kp++nsftz66hqXVgn7VZ19zGTU/kUlbbzDPfO50rZmYEuiQRCYCgmjB8cCrhxzs6xqzDPMbbvrMvD/bAe2N+djLzs5NZ/MFOfv7GFjYV1fDzq2Zwxvik7t/cj6obWsnNP8B9b29ja2ktv7/uFHKyErt/o4iEpKDqgWcld0z5ezevFICF00fQ3OYl3GOHlpw9EYvOGs+TN82h3eu49pHPuOuFdVQ1tJzw556o5rZ2fvFmHqf/4l1ueiKXggMNPHp9Dgt00FJkULP+PKklJyfH5eb2/tJoXq9j9k/foaqhlZiIMNb++AJ2ldeTEBPRp1edaWxp54H3trP4g10Mi4ngxvljqWtu47rTx5DRw+/ZUVbHvUu20NzmJSUuiphID87B8NhIEodEMi5lCLMyh5MQE4HX6444AFnV0EJRVRPVja28v62cd/NK2V5Wx1WzMrh2TibTRsYzRGdbigwaZrbKOZdz9PagSgGPx5g3PpnXNxQzangMUeFhTEnv+4N3MZFh3L1wMl86aST3vLSB/35rKwDPryzgF1fP4IKpacecb/3quiL+5S/riIkMIzMxlh2ltTS2tmNmVDW0cPA4aZjHGBodTl1TGxnDY/A6R2SYhz37Gw4dTI0IM6amx7P4W6dwodb3FpHDBFWAQ8d49esbivtkyKQ7U0fG89I/zKWstpm65ja+//QqFj25inkTkrhjwUQmpg6lqLqRVfmV5O45wMo9lZTVNtHa7jg1azgPfWP2oemNB3m9jsqGFraW1vLpzv1U1LUQHxNOYWUj4R6juc3LhdNGMCMjgXCPMXdCstY2EZFOBV0yzJ+QDMDo4f1zoV6PxxiR0BHCr//gTP68Yi/3L93GV3/36RGvSxkaxZysREYnxjI0Opyb5o8lOiKs089LiotiblwUc8cn90sbRCQ0BV2Aj06M5a6Fkzh7Yu/XFu+tiDAP18/N4spZGXy8o4LCykZS46OYNXo4oxNjdBq7iPSroAtwgO+fMyGg358QE8ElM9K7f6GIiB8F1TRCERH5OwW4iEiQUoCLiAQpBbiISJBSgIuIBCkFuIhIkFKAi4gEKQW4iEiQ6tfVCM2sHMjv5duTgeC9eOXxU3tDm9ob2vq6vWOcc184/bxfA/xEmFluZ8sphiq1N7SpvaGtv9qrIRQRkSClABcRCVLBFOCLA11AP1N7Q5vaG9r6pb1BMwYuIiJHCqYeuIiIHEYBLiISpIIiwM1soZltNbMdZvajQNfjD2a2x8w2mNlaM8v1bUs0s3fMbLvvz+GBrrO3zOwxMyszs42Hbeu0fdbhAd/+Xm9mswNXee900d6fmFmhbx+vNbNLDnvuHl97t5rZRYGpunfMbLSZLTOzPDPbZGa3+7aH5P49Rnv7f/865wb0DQgDdgLjgEhgHTA10HX5oZ17gOSjtv0S+JHv/o+AewNd5wm07yxgNrCxu/YBlwBvAgacDqwIdP191N6fAP/SyWun+n6uo4Cxvp/3sEC34Tjamg7M9t0fCmzztSkk9+8x2tvv+zcYeuBzgB3OuV3OuRbgWeCKANfUX64AnvDdfwK4MoC1nBDn3AfAgaM2d9W+K4A/uQ6fAcPMLKiuYddFe7tyBfCsc67ZObcb2EHHz31QcM4VO+dW++7XAnlABiG6f4/R3q74bf8GQ4BnAAWHPd7Hsf+ygpUD3jazVWa2yLctzTlXDB0/NEBqwKrzj67aF8r7/DbfsMFjhw2JhUx7zSwLmAWsYBDs36PaC/28f4MhwDu71Hsozn2c55ybDVwM3GpmZwW6oAAK1X3+MDAemAkUA/f5todEe80sDvgrcIdzruZYL+1kWyi0t9/3bzAE+D5g9GGPRwFFAarFb5xzRb4/y4CX6PgVq/Tgr5a+P8sCV6FfdNW+kNznzrlS51y7c84LPMLff40O+vaaWQQdYfa0c+5F3+aQ3b+dtTcQ+zcYAnwlkG1mY80sErgG+FuAa+pTZjbEzIYevA9cCGyko53X+152PfBKYCr0m67a9zfg277ZCqcD1Qd/FQ9mR43zXkXHPoaO9l5jZlFmNhbIBj7v7/p6y8wMeBTIc8796rCnQnL/dtXegOzfQB/R7eFR30voONK7E/i3QNfjh/aNo+Mo9Tpg08E2AknAu8B235+Jga71BNr4DB2/VrbS0SO5qav20fEr50O+/b0ByAl0/X3U3id97Vnv+0edftjr/83X3q3AxYGu/zjbOp+OIYH1wFrf7ZJQ3b/HaG+/71+dSi8iEqSCYQhFREQ6oQAXEQlSCnARkSClABcRCVIKcBGRIKUAFxEJUgpwEZEg9f8BiAYUgKplnl0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 48)                1200      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 98        \n",
      "=================================================================\n",
      "Total params: 1,418\n",
      "Trainable params: 1,418\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "agent.model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent.env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:new_tf]",
   "language": "python",
   "name": "conda-env-new_tf-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
